package extractorCode;


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class link1Edit {
	
	private final static int INITIAL_BUF_SIZE = 65536;
	public static void main(String args[]) throws Exception
	{
	
		int count=0;
	    boolean nextIndex=true;

	    
	    BufferedReader reader= new BufferedReader(new FileReader("c:\\keywords-link1.txt"));

	    //for all keywords...
	    String keyword;
	    int lastIndex;
	    while((keyword=reader.readLine())!=null)
	    {
	    	count=0;
	    	String[] splitarr=keyword.split("\t");
	    	String key = splitarr[0];
	    	int maxPage=Integer.parseInt(splitarr[1]);
	    	System.out.println(keyword);
	       if(maxPage>-1)
	       {
	    	   BufferedReader reader2= new BufferedReader(new FileReader("C:\\Users\\Anu\\Desktop\\Website1\\link1-links\\link1-"+key+".txt"));
	    	BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\Website1-links\\link1.2-"+key+".txt"));
	    	String link,date = "",title;
	    	while((link=reader2.readLine())!=null)
	    	{
	    		
	    		String[] splitarr2=link.split("\t");
		    	link = splitarr2[0];
		    	
	    	   //now fetch the page
		    	link=link.replace("href=","");
				link=link.replace("\"","");
				link=link.trim();
				//date=date.trim();
				System.out.println(link);
				//link="http://nasional.kompas.com/read/2008/01/17/11382753/anggota.pks.minta.kpi.awasi.film.naruto";
		    	String page= fetchPage(link);
		    	System.out.println(page);
		    	
		    	Matcher titleItem=Pattern.compile("<div class=\"judulisiberita\" style=\"margin:5px 0px 5px 0px;\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
				Matcher titleItem2=Pattern.compile("<div class=\"judul_artikel2011\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
				
				String toCheck="";
				/*if(splitarr2.length>1)
				{	date= splitarr2[1];
				
			    toCheck = date.substring(0,1);
				System.out.println("to check!!!! "+toCheck);
				System.out.println(isInteger(toCheck));
				
				if(!isInteger(toCheck))
				{
				   //access the link..
					Matcher dateItem2=Pattern.compile("<div class=\"tanggal\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
					
					Matcher dateItem3=Pattern.compile("<span class=\"c_abu01_kompas2011\">(.*?)</span>", Pattern.CASE_INSENSITIVE).matcher(page);
					
					while(dateItem2.find())
					{
						date= dateItem2.group(1);
						System.out.println(date);
					}
					
					while(dateItem3.find())
					{
						date= dateItem3.group(1);
						System.out.println(date);
					}
					
				}
				}*/
			    				    	
			    	  //access the link..
						Matcher dateItem2=Pattern.compile("<div class=\"tanggal\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
						
						Matcher dateItem3=Pattern.compile("<span class=\"c_abu01_kompas2011\">(.*?)</span>", Pattern.CASE_INSENSITIVE).matcher(page);
						
						while(dateItem2.find())
						{
							date= dateItem2.group(1);
							System.out.println(date);
						}
						
						while(dateItem3.find())
						{
							date= dateItem3.group(1);
							System.out.println(date);
						}
						
					
			    	
			    	
			    	
			    	
						String[] splitDate= date.split(",");
						if(splitDate.length>1)
						{ System.out.println("Splitting "+ splitDate[0]+" "+ splitDate[1]);
							date=splitDate[1];
			    	
						}
				
				
				if(link.contains(".pdf"))
				{
					String[] splitLink= link.split("/");
					String name= splitLink[splitLink.length-1];
					System.out.println(name);
					title= name.replaceAll(".pdf","");
					date= "Not provided";
					date=date.trim();
					wr.write(link+","+date+","+title+"\n");
					System.out.println(link+","+date+","+title+"\n");
					
					count++;
				}
				if(page.equals(""))
		    	{
		    		String[] splitPage= link.split("/");
		    		System.out.println(splitPage.length);
		    		title= splitPage[splitPage.length-1];
		    		date= date.trim();
					wr.write(link+","+date+","+title+"\n");
					System.out.println(link+","+date+","+title+"\n");
				
					count++;
		    	}
				else
				{
				while(titleItem.find())
				{
			//	System.out.println(newsItem.group(1));
				
				title= titleItem.group(1);
				date= date.trim();
				wr.write(link+","+date+","+title+"\n");
				System.out.println(link+","+date+","+title+"\n");
			
				count++;
				//System.out.println(text);
			//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
				}
				
				while(titleItem2.find())
				{
			//	System.out.println(newsItem.group(1));
				
				title= titleItem2.group(1);
				date= date.trim();
				wr.write(link+","+date+","+title+"\n");
				System.out.println(link+","+date+","+title+"\n");
				
				count++;
				//System.out.println(text);
			//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
				}
				}
	    	}

	    	
	   /* for(int i=0;i<=maxPage;i++)
		{	
		System.out.println("page"+i);	
		String page = (fetchPage("http://www.google.co.id/custom?hl=id&client=pub-9012468469771973&cof=FORID:9%3BAH:left%3BCX:Kompas%252Ecom%2520Search%2520%3BL:http://www.google.com/intl/id/images/logos/custom_search_logo_sm.gif%3BLH:30%3BLC:%230000ff%3BVLC:%23663399%3BDIV:%23336699%3B&cx=partner-pub-9012468469771973:8420067735&ad=n9&adkw=AELymgUT9jR5jGKDrI_lNNHP7w1SIAEWqsEzaYKKF2nLD-Ou9SOTsJEm_LRr12LUdL-jWy9ajkqxHsZf6vlvHxni_ZnBcPKXhqE7CIuaHlkn0o0D8Py5Xqw&channel=2570703154&boostcse=0&ei=HJHmTrqmMKHs2AW_pejRBA&q="+key+"&start="+(i*10)+"&sa=N"));
		//System.out.println(page);
		
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		Matcher newsItem = Pattern.compile("<div class=\"g\"><h2 class=\"r\"><a class=\"l\"\\s*(href=\"(.*?)\")", Pattern.CASE_INSENSITIVE).matcher( page );
		Matcher dateItem=Pattern.compile("<span class=\"s\">(.*?)<b>", Pattern.CASE_INSENSITIVE).matcher( page );
		Matcher titleItem=Pattern.compile("<div class=\"judul_artikel2011\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
		
		while(newsItem.find() && dateItem.find()&&titleItem.find())
		{
	//	System.out.println(newsItem.group(1));
		
		String text = newsItem.group(1);
		String date = dateItem.group(1);
		String title= titleItem.group(1);
		//check here if the date is valid
		String toCheck = date.substring(0,1);
		System.out.println("to check!!!! "+toCheck);
		System.out.println(isInteger(toCheck));
		if(!isInteger(toCheck))
		{
		   //access the link..
			text=text.replace("href=","");
			text=text.replace("\"","");
			String news= fetchPage(text);
			Matcher dateItem2=Pattern.compile("<div class=\"tanggal\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( news );
			
			Matcher dateItem3=Pattern.compile("<span class=\"c_abu01_kompas2011\">(.*?)</span>", Pattern.CASE_INSENSITIVE).matcher( news );
			
			while(dateItem2.find())
			{
				date= dateItem2.group(1);
				System.out.println(date);
			}
			
			while(dateItem3.find())
			{
				date= dateItem3.group(1);
				System.out.println(date);
			}
			
		}
		nextIndex= text.contains("<span>Next</span>");
		//System.out.println(text);
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		
		wr.write(text+","+date+","+title+"\n");
		System.out.println(text+"\t"+date+","+title+"\n");
		
		count++;
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		}*/
		System.out.println("The total number of links "+count);
		wr.close();
	       }
	    }
	}
	
	
	public static boolean isInteger( String input )  
	{  
	   try  
	   {  
	      Integer.parseInt( input );  
	      return true;  
	   }  
	   catch( Exception e)  
	   {  
	      return false;  
	   }  
	}  

	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			//URL url = new URL( urlString );
			
			
			 URL url = new URL(urlString);
            HttpURLConnection.setFollowRedirects(false);
            
             HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();

            urlConn.setRequestProperty("User-agent","Mozilla/5.0");
             urlConn.connect();
             BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));
             
			
			//BufferedInputStream reader = new BufferedInputStream(url.openStream());
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
